const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];

async function main() {
  let totalBlogs = [];
  const MAX_CNT = 50;
  const SLEEP_TIME = 200;

  const tab = 'imooc';
  const sub_tab = 'frontend';
  const url = 'https://www.imooc.com/article/fe';
  const fileName = `imooc_${sub_tab}.json`;


  const browser = await puppeteer.launch({headless: false, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });


  await page.exposeFunction('dateDiff', (nowDate, diffString) => {
    function _fmtDate(obj, needTime, justDate) {
      var date = new Date(obj);
      var y = 1900 + date.getYear();
      var m = "0" + (date.getMonth() + 1);
      var d = "0" + date.getDate();
      var h = "0" + date.getHours();
      var min = "0" + date.getMinutes();
      var s = "0" + date.getSeconds();
      if (justDate) {
        return y + "-" + m.substring(m.length - 2, m.length) + "-" + d.substring(d.length - 2, d.length);
      }
      if (needTime) {
        return y + "-" + m.substring(m.length - 2, m.length) + "-" + d.substring(d.length - 2, d.length) +
          ' ' + h.substring(h.length - 2, h.length) + ":00:00";
      } else {
        return y + "-" + m.substring(m.length - 2, m.length) + "-" + d.substring(d.length - 2, d.length) + ' 00:00:00';
      }
    }

    function _dateDiff(nowDate, diffString) {
      nowDate = new Date(nowDate);

      const minuteReg = /分钟前$/;
      const hourReg = /小时前$/;
      const dayReg = /天前$/;
      const monthReg = /月前$/;
      const yearReg = /年前$/;
      const yyyymmddReg = /年.+月.+日/;
      const mmddReg = /^\d+月.+日/;
      const todayReg = /^今天/;
      const yesterdayReg = /^昨天/;
      const beforeYesterdayReg = /^前天/;
      const exceptYear = /^(\d{0,9}\/{1}\d{0,9})\s(\d{0,9}\:{1}\d{0,9})/; // 匹配形如 03/15 15:19
      const slashReg = /^(\d{0,9}\/{1}\d{0,9}\/{1}\d{0,9})\s(\d{0,9}\:{1}\d{0,9})/; // 匹配形如 2018/03/15 15:19

      if (minuteReg.test(diffString)) {
        return _fmtDate(nowDate.getTime(), true);
      }

      if (hourReg.test(diffString)) {
        const hourTimestamp = Number(diffString.split('小时前')[0]) * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - hourTimestamp, true);
      }
      if (dayReg.test(diffString)) {
        const dayTimestamp = Number(diffString.split('天前')[0]) * 24 * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - dayTimestamp, false);
      }
      if (monthReg.test(diffString)) {
        const monthTimestamp = Number(diffString.split('月前')[0]) * 30 * 24 * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - monthTimestamp, false);
      }
      if (yearReg.test(diffString)) {
        const yearTimestamp = Number(diffString.split('年前')[0]) * 365 * 24 * 3600 * 1000;
        return _fmtDate(nowDate.getTime() - yearTimestamp, false);
      }
      if (yyyymmddReg.test(diffString)) {
        return _fmtDate(new Date(Date.parse(diffString.replace('年', '-').replace('月', '-').replace('日', ''))), false);
      }
      if (mmddReg.test(diffString)) {
        let yearStr = (new Date()).getFullYear() + '年';
        return _fmtDate(new Date(Date.parse((yearStr + diffString).replace('年', '-').replace('月', '-').replace('日', ''))), false);
      }
      if (todayReg.test(diffString)) {
        const timeString = diffString.split('今天')[1].replace(/(^\s*)|(\s*$)/g, "")
        return _fmtDate(nowDate.getTime(), false, true) + ' ' + timeString + ':00';
      }
      if (yesterdayReg.test(diffString)) {
        const timeString = diffString.split('昨天')[1].replace(/(^\s*)|(\s*$)/g, "")
        const timestamp = nowDate.getTime() - 24 * 3600 * 1000;
        return _fmtDate(timestamp, false, true) + ' ' + timeString + ':00';
      }
      if (beforeYesterdayReg.test(diffString)) {
        const timeString = diffString.split('前天')[1].replace(/(^\s*)|(\s*$)/g, "")
        const timestamp = nowDate.getTime() - 2 * 24 * 3600 * 1000;
        return _fmtDate(timestamp, false, true) + ' ' + timeString + ':00';
      }
      if (exceptYear.test(diffString)) {
        const dateString = (1900 + nowDate.getYear()) + '-' + diffString.replace(/\//g, "-") + ':00';
        return dateString;
      }
      if (slashReg.test(diffString)) {
        const dateString = diffString.replace(/\//g, "-") + ':00';
        return dateString;
      }
      return '时间格式有误';
    }

    return _dateDiff(nowDate, diffString);
  });


  await page.goto(url);
  await page.waitFor('#main > div > div.centerlist.l > div > div.articleCon.js-usercard-box > div');


  let preScrollHeight = 0;
  let scrollHeight = -1;
  let cnt = 0;
  let retry_cnt = 0;
  while (cnt < MAX_CNT) {
    // 详情信息是根据滚动异步加载，所以需要让页面滚动到屏幕最下方，通过延迟等待的方式进行多次滚动
    let scrollH1 = await page.evaluate(async () => {
      let h1 = document.body.scrollHeight;
      window.scrollTo(0, h1);
      return h1;
    });

    // 延迟等待时间
    await page.waitFor(SLEEP_TIME);

    let scrollH2 = await page.evaluate(async () => {
      return document.body.scrollHeight;
    });
    let scrollResult = [scrollH1, scrollH2];
    preScrollHeight = scrollResult[0];
    scrollHeight = scrollResult[1];

    if (preScrollHeight == scrollHeight) {
      retry_cnt++;
      if (retry_cnt > 3) { // 如果重试次数大于retry_cnt次, 认为发生异常, 进入容错处理

        // 容错处理
        await page.evaluate(x => {
          window.scrollTo(0, x - 1000)
        }, scrollHeight);
        await page.waitFor(1000);
        await page.evaluate(x => {
          window.scrollTo(0, x)
        }, scrollHeight);
        retry_cnt = 0;
      }
    } else {
      retry_cnt = 0;
      cnt++;
    }
    console.log(scrollResult, cnt);
  }

  totalBlogs = await collectBlogs(page, tab, sub_tab);

  let writerStream = fs.createWriteStream(`crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');


  // browser.close();
};

async function collectBlogs(page, tab, sub_tab) {
  let blogs = await page.$$eval('#main > div > div.centerlist.l > div > div.articleCon.js-usercard-box > div', async (eles, tab, sub_tab) => {
    console.log('eles length: ', eles.length);
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('div.list-content.l > a > p').innerText;
      let content = ele.querySelector('div.list-content.l > a').href;
      tmpBlogs.push({
        title: title,
        content: content,
        tab: tab,
        sub_tab: sub_tab
      });
    });
    return tmpBlogs;
  }, tab, sub_tab);
  return blogs;
}

main();


